"""
预处理数据，保留主干
"""
from multiprocessing import Pool
import jieba.posseg
import re

keep = ["n", "f", "s", "nr", "ns", "nt", "nw", "nz",
        "v", "vd", "vn", "t", "m", "q", "TIME", "ORG", "LOC", "PER", "w"]
set_not = ["不", "非", "有", "无", "莫", "高度"]

jieba.enable_paddle()  # 开启paddle模式


def keep_main_content(line):
    set_l = line.strip().split("\t")
    if bool(re.search("[-\d()*+/=\.]{15,}", set_l[1])):
        return
    words = jieba.posseg.cut(set_l[1], use_paddle=True)
    source = []
    target = []
    for item in words:
        target.append(item.word)
        if item.flag in keep or item.word in set_not:
            source.append(item.word)
    source = " ".join(source)
    target = " ".join(target)
    return source + "\t" + target + "\n"


def handle(in1, out1):
    fp = open(in1, 'r', encoding='utf-8')
    fout = open(out1, 'a', encoding='utf-8')
    pool = Pool(processes=4)
    lines = fp.readlines()
    result = pool.map(keep_main_content, lines)
    print('Mark~~~~~~~~~~~~~~~')
    pool.close()
    pool.join()
    print('sub-process done')
    print(len(result))
    fout.writelines(result)


if __name__ == "__main__":
    # infile = '../out3.tsv'
    # outfile = '../keep_content_split.tsv'
    infile = 'g:/1.txt'
    outfile = 'g:/keep_content_bottom.tsv'
    handle(infile, outfile)
